# 数据准备
import pandas as pd
from sklearn.model_selection import train_test_split
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# 读取微博情感分析数据集
df = pd.read_csv('/kaggle/input/sentiment-classification/nCoV_100k_train.labled.csv')

# 仅保留文本内容和情感标签列
df = df[['微博中文内容', '情感倾向']]
df = df.rename(columns={'微博中文内容': 'text', '情感倾向': 'label'})
print(df)
# 数据清洗
print(df.label.value_counts())
print(df.label.value_counts() / df.shape[0] * 100)
plt.figure(figsize=(8, 4))
sns.countplot(x='label', data=df)
plt.show()
df.drop(df[(df.label == '4') |
           (df.label == '-') |
           (df.label == '·') |
          (df.label == '-2') |
          (df.label == '10') |
           (df.label == '9')].index, inplace=True, axis=0)
df.reset_index(inplace=True, drop=True)
print(df.value_counts())
sns.countplot(x='label', data=df)
plt.show()
# 检查数据中是否存在空值
print(df.isnull().sum())
# 删除含有空值的行
df.dropna(axis=0, how='any', inplace=True)
df.reset_index(inplace=True, drop=True)
print(df.isnull().sum())
# 检查是否有完全重复的数据
print(df.duplicated().sum())
print(df[df.duplicated()==True])
# 删除完全重复的数据
index = df[df.duplicated() == True].index
df.drop(index, axis=0, inplace=True)
df.reset_index(inplace=True, drop=True)
print(df.duplicated().sum())
# 处理文本相同但标签不同的数据
print(df['text'].duplicated().sum())
print(df[df['text'].duplicated() == True])
# 查看具体重复样本

print(df[df['text'] == df.iloc[1473]['text']])
print(df[df['text'] == df.iloc[1814]['text']])

# 删除文本相同但标签不同的数据
index = df[df['text'].duplicated() == True].index
df.drop(index, axis=0, inplace=True)
df.reset_index(inplace=True, drop=True)
# 验证处理结果
print(df['text'].duplicated().sum())  # 0
print(df)
# 检查清洗后数据的形状和索引
print("======data-clean======")
print(df.tail())
print(df.shape)

# 统计文本长度分布
print(df['text'].str.len().sort_values())
# 将数据集拆分为训练集60%、验证集20%和测试集20%
train, test = train_test_split(df, test_size=0.2)
train, val = train_test_split(train, test_size=0.25)
print(train.shape)
print(test.shape)
print(val.shape)

train.to_csv('/kaggle/working/train.csv', index=None)
val.to_csv('/kaggle/working/val.csv', index=None)
test.to_csv('/kaggle/working/test.csv', index=None)